# -*- coding: utf-8 -*-
# Author: Sun Jiawei
# E-mail: sunjiawei@tbea.com
#
import re
import time
import random
import requests
import warnings
import datetime
import pandas as pd

from tqdm import tqdm
from bs4 import BeautifulSoup
# warnings.filterwarnings('ignore')
warnings.warn('Deprecated warning: use `coal_and_gas_price.py` instead.')
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36',
    'Cookie': 'Hm_lvt_1c4432afacfa2301369a5625795031b8=1649637881; _last_loginuname=tbeasjw; href=https://coal.mysteel.com/steamcoal/; accessId=5d36a9e0-919c-11e9-903c-ab24dbab411b; pageViewNum=5; _rememberStatus=false; _login_token=7efefe6344d3faf0611c0db0636cf0e2; _login_uid=7326688; _login_mid=7954258; _login_ip=222.90.211.2; 7efefe6344d3faf0611c0db0636cf0e2=33=5&22=5&11=5&34=5&13=5&35=5&36=5&37=5&38=5&1=5&2=5&4=10&40=5&41=5&42=5&31=5&32=5&catalog=020105,020206,0204,0224,0213,1002,0220,1006,0202,0222,0223,0205,0209,1001,1007,010205,010202,0203,0224,02,0201,0223,0205,0222; _last_ch_r_t=1649724904510; Hm_lpvt_1c4432afacfa2301369a5625795031b8=1649725105; _dd_s=logs=1&id=713e2ab6-a2e8-4fa3-a8a0-98731f2dc1de&created=1649724343397&expire=1649726964454'
}


def handle_tag_list(tag_list: list):
    region_list = []
    coal_kind = []
    price_list = []

    for i in range(len(tag_list)):
        if tag_list[i].contents[0] == '地区':
            for j in range(i + 1, len(tag_list)):
                if tag_list[j].contents[0] != '煤种':
                    region_list.append(tag_list[j].contents[0])
                else:
                    i = j
                    break
        if tag_list[i].contents[0] == '煤种':
            for j in range(i + 1, len(tag_list)):
                if tag_list[j].contents[0] != '价格':
                    coal_kind.append(tag_list[j].contents[0])
                else:
                    i = j
                    break
        if tag_list[i].contents[0] == '价格':
            for j in range(i + 1, len(tag_list)):
                if tag_list[j].contents[0] != '涨幅':
                    price_list.append(tag_list[j].contents[0])
                else:
                    i = j
                    break
        break
    return region_list, coal_kind, price_list


def get_all_detail_urls(base_url: str):
    """给定一个目录页面,返回该页面上所有记录价格数据的子页面的链接"""
    ret = {}
    html = requests.get(base_url, headers=headers)
    soup = BeautifulSoup(html.content)
    target_tags = soup.find_all(title=re.compile(r'\d+日全国主要城市动力煤价格汇总'))
    for tag in target_tags:
        target_url = tag.attrs['href']
        year, md = target_url.split('/')[4:6]
        year = '20' + year
        month = md[:2]
        day = md[2:]
        date = year + '-' + month + '-' + day
        ret[date] = target_url
    return ret


def get_price_detail(data_url: str):
    """给定一个价格页面的链接,找到对应的数据"""
    # 随机睡3-10秒,以免被封
    sleep_time = 7 * random.random() + 3
    print(f"sleep {sleep_time:.2f}s")
    time.sleep(sleep_time)
    html = requests.get(data_url, headers=headers)
    soup = BeautifulSoup(html.content, 'lxml')
    tag_list = soup.find_all('td')
    r, c, p = handle_tag_list(tag_list)
    r_c = []
    for i in range(len(c)):
        r_c.append(r[i] + '\\' + c[i])
    return r_c, p


if __name__ == '__main__':
    df = pd.DataFrame()
    for i in tqdm(range(1, 18)):
        base_link = f'https://coal.mysteel.com/price/p-10034--050201--{i}.html'
        ret = get_all_detail_urls(base_link)
        data = []
        index = []
        for key in tqdm(ret):
            if key == datetime.datetime.today().strftime('%Y-%m-%d'):
                # 当天的数据可能不准,去掉
                continue
            date = key
            url = ret[key]
            columns, data_ = get_price_detail(url)
            _df = pd.DataFrame([data_], index=[date], columns=columns)
            df = df.append(_df)

    df.to_csv(f'data.csv')
